{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bert_Blend-CNN.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "UahhCDlxtpWr"
      },
      "source": [
        "# -*- coding:utf-8 -*-\n",
        "# bert融合textcnn思想的Bert+Blend-CNN\n",
        "# model: Bert+Blend-CNN\n",
        "# date: 2021.10.11 18:06:11\n",
        "\n",
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.utils.data as Data\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "import transformers\n",
        "from transformers import AutoModel, AutoTokenizer\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dL4eT_MTS9JY"
      },
      "source": [
        "train_curve = []\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
      ],
      "execution_count": 63,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZdVB3Lt6TAEs"
      },
      "source": [
        "# # 定义一些参数，模型选择了最基础的bert中文模型\n",
        "batch_size = 2\n",
        "epoches = 100\n",
        "model = \"bert-base-chinese\"\n",
        "hidden_size = 768\n",
        "n_class = 2\n",
        "maxlen = 8\n",
        "\n",
        "encode_layer=12\n",
        "filter_sizes = [2, 2, 2]\n",
        "num_filters = 3"
      ],
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gQ3SK8_rTFGX"
      },
      "source": [
        "# data，构造一些训练数据\n",
        "sentences = [\"我喜欢打篮球\", \"这个相机很好看\", \"今天玩的特别开心\", \"我不喜欢你\", \"太糟糕了\", \"真是件令人伤心的事情\"]\n",
        "labels = [1, 1, 1, 0, 0, 0]  # 1积极, 0消极."
      ],
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8eBhZbDTJSz"
      },
      "source": [
        "class MyDataset(Data.Dataset):\n",
        "  def __init__(self, sentences, labels=None, with_labels=True,):\n",
        "    self.tokenizer = AutoTokenizer.from_pretrained(model)\n",
        "    self.with_labels = with_labels\n",
        "    self.sentences = sentences\n",
        "    self.labels = labels\n",
        "  def __len__(self):\n",
        "    return len(sentences)\n",
        "\n",
        "  def __getitem__(self, index):\n",
        "    # Selecting sentence1 and sentence2 at the specified index in the data frame\n",
        "    sent = self.sentences[index]\n",
        "\n",
        "    # Tokenize the pair of sentences to get token ids, attention masks and token type ids\n",
        "    encoded_pair = self.tokenizer(sent,\n",
        "                    padding='max_length',  # Pad to max_length\n",
        "                    truncation=True,       # Truncate to max_length\n",
        "                    max_length=maxlen,  \n",
        "                    return_tensors='pt')  # Return torch.Tensor objects\n",
        "\n",
        "    token_ids = encoded_pair['input_ids'].squeeze(0)  # tensor of token ids\n",
        "    attn_masks = encoded_pair['attention_mask'].squeeze(0)  # binary tensor with \"0\" for padded values and \"1\" for the other values\n",
        "    token_type_ids = encoded_pair['token_type_ids'].squeeze(0)  # binary tensor with \"0\" for the 1st sentence tokens & \"1\" for the 2nd sentence tokens\n",
        "\n",
        "    if self.with_labels:  # True if the dataset has labels\n",
        "      label = self.labels[index]\n",
        "      return token_ids, attn_masks, token_type_ids, label\n",
        "    else:\n",
        "      return token_ids, attn_masks, token_type_ids"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q-fhfJi7xkXd"
      },
      "source": [
        "train = Data.DataLoader(dataset=MyDataset(sentences, labels), batch_size=batch_size, shuffle=True, num_workers=1)"
      ],
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aqjKaE6-UPE1"
      },
      "source": [
        "class TextCNN(nn.Module):\n",
        "  def __init__(self):\n",
        "    super(TextCNN, self).__init__()\n",
        "    self.num_filter_total = num_filters * len(filter_sizes)\n",
        "    self.Weight = nn.Linear(self.num_filter_total, n_class, bias=False)\n",
        "    self.bias = nn.Parameter(torch.ones([n_class]))\n",
        "    self.filter_list = nn.ModuleList([\n",
        "      nn.Conv2d(1, num_filters, kernel_size=(size, hidden_size)) for size in filter_sizes\n",
        "    ])\n",
        "\n",
        "  def forward(self, x):\n",
        "    # x: [bs, seq, hidden]\n",
        "    x = x.unsqueeze(1) # [bs, channel=1, seq, hidden]\n",
        "    \n",
        "    pooled_outputs = []\n",
        "    for i, conv in enumerate(self.filter_list):\n",
        "      h = F.relu(conv(x)) # [bs, channel=1, seq-kernel_size+1, 1]\n",
        "      mp = nn.MaxPool2d(\n",
        "        kernel_size = (encode_layer-filter_sizes[i]+1, 1)\n",
        "      )\n",
        "      # mp: [bs, channel=3, w, h]\n",
        "      pooled = mp(h).permute(0, 3, 2, 1) # [bs, h=1, w=1, channel=3]\n",
        "      pooled_outputs.append(pooled)\n",
        "    \n",
        "    h_pool = torch.cat(pooled_outputs, len(filter_sizes)) # [bs, h=1, w=1, channel=3 * 3]\n",
        "    h_pool_flat = torch.reshape(h_pool, [-1, self.num_filter_total])\n",
        "    \n",
        "    output = self.Weight(h_pool_flat) + self.bias # [bs, n_class]\n",
        "\n",
        "    return output"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAqolyEvTNYJ"
      },
      "source": [
        "# model\n",
        "class Bert_Blend_CNN(nn.Module):\n",
        "  def __init__(self):\n",
        "    super(Bert_Blend_CNN, self).__init__()\n",
        "    self.bert = AutoModel.from_pretrained(model, output_hidden_states=True, return_dict=True)\n",
        "    self.linear = nn.Linear(hidden_size, n_class)\n",
        "    self.textcnn = TextCNN()\n",
        "    \n",
        "  def forward(self, X):\n",
        "    input_ids, attention_mask, token_type_ids = X[0], X[1], X[2]\n",
        "    outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids) # 返回一个output字典\n",
        "    # 取每一层encode出来的向量\n",
        "    # outputs.pooler_output: [bs, hidden_size]\n",
        "    hidden_states = outputs.hidden_states # 13*[bs, seq_len, hidden] 第一层是embedding层不需要\n",
        "    cls_embeddings = hidden_states[1][:, 0, :].unsqueeze(1) # [bs, 1, hidden]\n",
        "    # 将每一层的第一个token(cls向量)提取出来，拼在一起当作textcnn的输入\n",
        "    for i in range(2, 13):\n",
        "      cls_embeddings = torch.cat((cls_embeddings, hidden_states[i][:, 0, :].unsqueeze(1)), dim=1)\n",
        "    # cls_embeddings: [bs, encode_layer=12, hidden]\n",
        "    logits = self.textcnn(cls_embeddings)\n",
        "    return logits"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_E6qpfSATZd1",
        "outputId": "baa36957-5382-42f5-b0d0-9d334070732b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "bert_blend_cnn = Bert_Blend_CNN().to(device)"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertModel: ['cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
            "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QblwR6DexAFe"
      },
      "source": [
        "optimizer = optim.Adam(bert_blend_cnn.parameters(), lr=1e-3, weight_decay=1e-2)\n",
        "loss_fn = nn.CrossEntropyLoss()"
      ],
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bBzQBHt8xbKm",
        "outputId": "a43c4a60-17af-4d63-f3c4-afd329aa8605",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# train\n",
        "sum_loss = 0\n",
        "total_step = len(train)\n",
        "for epoch in range(epoches):\n",
        "  for i, batch in enumerate(train):\n",
        "    optimizer.zero_grad()\n",
        "    batch = tuple(p.to(device) for p in batch)\n",
        "    pred = bert_blend_cnn([batch[0], batch[1], batch[2]])\n",
        "    loss = loss_fn(pred, batch[3])\n",
        "    sum_loss += loss.item()\n",
        "\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    if epoch % 10 == 0:\n",
        "      print('[{}|{}] step:{}/{} loss:{:.4f}'.format(epoch+1, epoches, i+1, total_step, loss.item()))\n",
        "  train_curve.append(sum_loss)\n",
        "  sum_loss = 0"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[1|100] step:1/3 loss:1.3587\n",
            "[1|100] step:2/3 loss:0.5860\n",
            "[1|100] step:3/3 loss:1.3804\n",
            "[11|100] step:1/3 loss:0.7330\n",
            "[11|100] step:2/3 loss:0.9912\n",
            "[11|100] step:3/3 loss:0.5007\n",
            "[21|100] step:1/3 loss:0.6944\n",
            "[21|100] step:2/3 loss:0.6947\n",
            "[21|100] step:3/3 loss:0.6936\n",
            "[31|100] step:1/3 loss:0.7441\n",
            "[31|100] step:2/3 loss:0.6923\n",
            "[31|100] step:3/3 loss:0.6735\n",
            "[41|100] step:1/3 loss:0.6875\n",
            "[41|100] step:2/3 loss:0.7020\n",
            "[41|100] step:3/3 loss:0.6898\n",
            "[51|100] step:1/3 loss:0.4228\n",
            "[51|100] step:2/3 loss:0.2038\n",
            "[51|100] step:3/3 loss:0.0154\n",
            "[61|100] step:1/3 loss:0.0064\n",
            "[61|100] step:2/3 loss:0.0058\n",
            "[61|100] step:3/3 loss:0.0060\n",
            "[71|100] step:1/3 loss:0.0039\n",
            "[71|100] step:2/3 loss:0.0036\n",
            "[71|100] step:3/3 loss:0.0039\n",
            "[81|100] step:1/3 loss:0.0021\n",
            "[81|100] step:2/3 loss:0.0020\n",
            "[81|100] step:3/3 loss:0.0020\n",
            "[91|100] step:1/3 loss:0.0029\n",
            "[91|100] step:2/3 loss:0.0025\n",
            "[91|100] step:3/3 loss:0.0256\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QkDiBPY3xhxK",
        "outputId": "f1cfed2a-c23d-4d2e-8269-90c1437c7831",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# test\n",
        "bert_blend_cnn.eval()\n",
        "with torch.no_grad():\n",
        "  test_text = ['我不喜欢打篮球']\n",
        "  test = MyDataset(test_text, labels=None, with_labels=False)\n",
        "  x = test.__getitem__(0)\n",
        "  x = tuple(p.unsqueeze(0).to(device) for p in x)\n",
        "  pred = bert_blend_cnn([x[0], x[1], x[2]])\n",
        "  pred = pred.data.max(dim=1, keepdim=True)[1]\n",
        "  if pred[0][0] == 0:\n",
        "    print('消极')\n",
        "  else:\n",
        "    print('积极')"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "消极\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qELdtXhu2qOw",
        "outputId": "b8bfc850-d8e8-4c5b-b624-7ae77b0b7bfb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        }
      },
      "source": [
        "pd.DataFrame(train_curve).plot() # loss曲线"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fee604a25d0>"
            ]
          },
          "metadata": {},
          "execution_count": 75
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXSc9X3v8fd3di2jxZJsS7LBJl7AhtgQEyBkJyyhCSG3CZc0S5f0uu0lN0vb01Pa2yWnSZOmaVKSNDnlhjZbA03bUBNKSQhLyEIMMpDgDWywsSXLtmRrl2b/3T9mRpZsyZJsyfPMM5/XQcea0TMzv4fH56Ovf89vMeccIiLiXYFSN0BERE5PQS0i4nEKahERj1NQi4h4nIJaRMTjQgvxps3NzW7FihUL8dYiIr60bdu2Xudcy1Q/W5CgXrFiBR0dHQvx1iIivmRmL0/3M3V9iIh4nIJaRMTjFNQiIh63IH3UIiKlkE6n6ezsJJFIlLop04rFYixbtoxwODzr1yioRcQ3Ojs7icfjrFixAjMrdXNO4Zzj2LFjdHZ2snLlylm/Tl0fIuIbiUSCpqYmT4Y0gJnR1NQ054pfQS0ivuLVkC46k/aVRVBve/k4u7oHS90MEZGSKIug/rP/3MHnHnqh1M0QEZmVBx98kLVr17Jq1So+/elPn/X7zepmopntB4aALJBxzm0660+eg7F0ltFU5lx+pIjIGclms9x222089NBDLFu2jMsvv5ybbrqJdevWnfF7zqWifpNzbuO5DmmAZDpLMp071x8rIjJnTz75JKtWreKCCy4gEolw6623smXLlrN6z7IYnpfI5EhksqVuhoiUkY9/bwc7D83vva11bXX8xdvXn/aYrq4uli9fPv542bJlbN269aw+d7YVtQN+YGbbzGzzVAeY2WYz6zCzjp6enrNq1MmS6SwJVdQiUqFmW1G/1jnXZWaLgYfMbLdz7vGJBzjn7gTuBNi0adO87pibzORIqqIWkTmYqfJdKO3t7Rw8eHD8cWdnJ+3t7Wf1nrOqqJ1zXYU/jwL3Aq8+q0+dg0w2RybnVFGLSFm4/PLL2bNnD/v27SOVSnHPPfdw0003ndV7zhjUZlZjZvHi98B1wPaz+tQ5SGbyAZ1Iq6IWEe8LhUJ86Utf4vrrr+eiiy7illtuYf36s6vuZ9P1sQS4tzCbJgR82zn34Fl96hwUg7r4p4iI1914443ceOON8/Z+Mwa1c+4lYMO8feIcFfumU5kcuZwjEPD29FARkfnm+ZmJE/umU1lV1SJSeTwf1BNHe6ifWkRm4ty8Djqbd2fSPu8H9YSKWiM/ROR0YrEYx44d82xYF9ejjsVic3qd52cmTryJqLHUInI6y5Yto7Ozk/medDefiju8zIXng3pid4cqahE5nXA4PKedU8qF97s+MhO7PlRRi0jlKYOgzk74XhW1iFQe7wd1WhW1iFQ2zwd1QhW1iFQ4zwe1KmoRqXTeD2rdTBSRClcGQa2uDxGpbJ4P6oS6PkSkwnk+qJOZLJFQoPC9KmoRqTxlENQ56mIhzPJ7J4qIVBrvB3U6RzQUJBoKkFBFLSIVyPNBnchkiYYDRENBVdQiUpE8H9TJdI5YKEgsHNCiTCJSkbwf1IWKOhYOTpqlKCJSKcogqHNEQwGiocCkWYoiIpXC+0GdzhINBVVRi0jF8n5QZ3LEwqqoRaRylUVQq6IWkUrm/aBOZwt91EGN+hCRiuT5oE5kcvlx1OGANrcVkYrk+c1tk+lsfhx1KKc+ahGpSJ6vqJOFijo/4UUVtYhUHk9X1JlsjkzOFdb6yGn1PBGpSJ6uqIvBHA2pohaRylUWQR0LB4mGgmRyjkxWVbWIVJZZB7WZBc3sGTO7fyEbNFFxlEexos4/p6AWkcoyl4r6I8CuhWrIVIqjPIqLMoG24xKRyjOroDazZcCvAF9d2OZMlhivqPMbB4AqahGpPLOtqP8e+CNg2pQ0s81m1mFmHT09PfPSuGJFHVNFLSIVbMagNrO3AUedc9tOd5xz7k7n3Cbn3KaWlpZ5adyJUR/B8T5qTSMXkUozm4r6auAmM9sP3AO82cy+taCtKph4MzEaCk56TkSkUswY1M65251zy5xzK4BbgUecc+9b8JZxonqOhoJEVVGLSIXy9MzEYvUcCwdIZVVRi0hlmlNQO+ceAx5bkJZMIZme2Eed/14VtYhUGo9X1CfGUadzqqhFpDJ5OqiLQ/GioQDpbGEctSpqEakwng7qiWt9ZHIOQNtxiUjF8fiiTPlQjgRPTHhRRS0ilcbjQZ0jEgwQCNj4FHLNTBSRSuPpoE6ks+Pjp8PBAMGAqetDRCqOp4M6mcmNz0iE/E1FdX2ISKXxdlCnc+NdHpC/qaiKWkQqjaeDOpE50fUBEAsFNOFFRCqOp4M6mc4Rm9j1EQ5qPWoRqTjeDuqTKupoaHYb3GayOQbG0gvZNBGRc8bjQX1qH/VsKuqvP/Ey1/zdYzjnFrJ5IiLnhLeDOp09ZdTHbCrqfb3D9A6nGNOYaxHxAW8HdSY3vrMLFCrqWYRv32i+22M4kVmwtomInCueD+pTxlHPouujfzQFwFBSQS0i5c/bQZ3OnjqOejYV9Ui+oh5SRS0iPuDpoE5kcpPHUYdnN466WFGr60NE/MDTQZ1MZyePow4FZ7VxwHgfdVJD9ESk/Hk7qM+gok6ks+OjPQZVUYuID3gmqHM5x0fveYbvPt0J5CetZHLupJuJ+Yr6dOOj+0dPVNHq+hARP/BMUAcCxuN7enlqfx8wYb/E0OSKOucgnZ0+qPsK/dMAwxr1ISI+4JmgBmhriHGofwyYvA1XUfH7062gNzGohxLqoxaR8uetoK6vonugGNQnNrYtKn5/ujWpJ3V9qKIWER/wVlA3VNHVN4ZzbjyMJy3KVKyoTzOWulhRV0eCGkctIr7gqaBub6hiJJVlMJEZ796YeDNxfIPb08xOLFbU5y2qVlCLiC94KqhbG2IAHOofG6+oY+FTuz6KFfUTLx5jy7Ndk96jbyRFVThIc21UXR8i4gueCuq2hiqgENTjoz6mqqjzQX3Hwy/wqQd2T3qPvtE0jdVhaqMhDc8TEV/wVFC3F4N6IDHjzUTnHDsPDXJ0KEEme6IrpH80RUN1hNpYSKM+RMQXPBXULbVRwkHjUP/Y+AzEqSrqRCZLV/8Yg4kMOQdHhpLjx/SNpmisCROPhbR6noj4gqeCOhAwltTFCl0f+Yo6dtIUcoBEOsfOQ4PjzxfHXkP+ZmJDdYR4NMRwMqNdXkSk7M0Y1GYWM7MnzewXZrbDzD6+kA1qa6iadDPx5CnkkO+j3tk9dVD3jabyfdSxEM7BSEq7vIhIeZtNRZ0E3uyc2wBsBG4wsysXqkHtDVUc6k+cuJl4mop6aV1+lEj3QALIrxcyMJamsTpCPBYGtN6HiJS/GYPa5Q0XHoYLXwvWn9DWEOPwYILRVD5gJy5zWvw+mc5X1JtWNBKPheguVNSDiTQ5R/5mYjQEaKlTESl/s+qjNrOgmT0LHAUecs5tneKYzWbWYWYdPT09Z9ygtoYqsjnHweOjwMkzE/PfHx1K0tk3xrq2Otrqq+jqz1fUxXWoi10foKVORaT8zSqonXNZ59xGYBnwajO7eIpj7nTObXLObWppaTnjBrXV54fovdQ7AkAkOHF4Xr6ifuZAPwDrWutoa4iNrw9SnD7eWB2hrhDU6voQkXI3p1Efzrl+4FHghoVpzolJL/t6R4gEAwQCNv6zYMAIB41fdBaCuq2O1oaq8T7q4hZcDdVhaqOFPmoN0RORMjebUR8tZtZQ+L4KuBbYffpXnbm2wjTyrv6xSd0eRbFQkNFUlubaKIvjMdrqYxwfSTGWyo5vattYmPACWupURMpfaBbHtAJfN7Mg+WD/jnPu/oVqUDxWmKySyEwamlcUDQcZSmZY11YHnKjAuwfGJnV9WCHjtTCTiJS7GYPaOfdL4NJz0JZxbfVVPJ8YmjR9vKj43LrWfFC31heDOkH/aJqAQTwWGh+Woq4PESl3npqZWFTs/piy66Pw3ImK+kRXSV9hnY9AwAgGjBqtSS0iPuDRoM5XybEpuj6K630UK+ql9YVJL/2JwvTx8Pix8VhYoz5EpOx5OqinqqijoQCxcICVzTWFx/m1p4t91I3VkfFja2MhdX2ISNnzZFAXlzudqo+6uTbKhmUNBCcM22triBW6PvJrURfVRkMMatSHiJS52Yz6OOdaC90ZU436+JtffSW5k1bEa62P8WLPCCPJDOsLfdeQv6moilpEyp0nK+rxPuopuj4aayI01UZPOb67f2x85byieEy7vIhI+fNkRb20PobZ1BX1VNrqq8aXM22Y2EcdDWnUh4iUPU9W1OFg/mZh80mV83SKm+ICk24mxmNhdX2ISNnzZEUN8K+br6I6MsuKutBVApxyM3E4mSGXc5PWDBERKSeerKgBWuJRaqKz+z1SXHEPJnd9xIsr6KVUVYtI+fJsUM9FSzxKqFAxN9ZMvpkIWupURMqbL4I6WNgUFyb3UWupUxHxA18ENZxY82PiFHItdSoifuCboG6tr6I6Epw0pC8+HtSqqEWkfHl21Mdc3bJpOWuW1E56Lj6+wa2CWkTKl2+C+rWrm3nt6uZJz9WqohYRH/BN18dU4rHCzUQFtYiUMV8HdXU4iJluJopIefN1UAcCRm0kxJD6qEWkjPk6qGHhVtB7sWeYgVFV6iKy8Hwf1LWx+V9BbyyV5eYv/ZQ/v2/7vL6viMhU/B/U0ak3D9hzZIgPfu0pfrDjMO6kjQhm8vDuIwwlM/xgxxFG1K0iIgvM90Edj4VP6aMeSWb4nW9t4+HdR9n8zW28/64nef7w0Kzf875nDxEJBhhLZ/nhriPz3WQRkUl8H9T5ro8TfcnOOf7k3ufY3zvCNz/4av7i7et4rmuAG+54nHd95Wd8+bG9vHBk+tAeGEvz2PM9/NoV59FaH+O+Zw+di9MQkQrm+6CORyffTLz7yYNsefYQH3vLGl63uoXfvHolj/3hG/nINatJZnJ85sHnue7zj/ONJ/ZP+X7f33GYVDbHzZe28/YNbTy+p4f+0dS5ORkRqUj+D+rCBreJdJa7nzzAX35vB69f08Jtb1o1fkxjTYSPvmUN3/s/r2Xrn1zDG9e28In7d7Gre/CU97vv2UOc31TNhmX13LShjXTW8d/bD5/LU5rWXPvaRaQ8+D6oa6NhRlNZXveZR7n9u89xUWsdn79lw7Q7viypi/F3795AQ3WYD9/9DGOFvRgBjg4l+NmLvbz9lW2YGevb6riguYYtz3adq9OZ1vauAS7/5A95/11b2Xno1F8wIlK+fLPWx3SWL8rv/nLh0jh3/M+NXPWKJsxOvy1XU22Uz92ykffdtZVPPrCTT9x8CQAP/LKbnIN3bGwDwMx4+4Y2vvDIHg4PJFhaH5vy/YaTGfYcGWIwkaGpJkJLPMqimgjh4OTfk845xtJZqiNzuyw7Dw3yvru2Eg0F+GXnAL/yxR/zzkvb+a2rV7K+rQ4zwznHk/uO842fv8zRwQSLaiIsqonQVl/Fxe31rG+rY3Hd1O0XkdKyhfjn8qZNm1xHR8e8v++ZyOYcfaOpWW+UO9GnHtjFPz7+Eq9esYi2hhjPHOynKhzkwY++fvyYF3uGuebvfsSbL1xMW0OMkWSWkWSGsXT+zyODSbr6x05571DAuKClhrVL61hUHWbX4SF2HRpkOJXh0uUNXLtuKa9b3Ty+HdloKsMzB/p5ct9xdnYPckl7PdevX8LS+ip+62tPEQ0FuGfzlTRURfjyY3v555/tJ5XJsaQuyutWt7Dz0CA7uwdpqA5z4dI4/aNpjo2k6BlKjrepKhykKhIkFgrQWBNh7dI4Fy2tY3FdlO6BBF19Y3QPJOgdTnJsJEkinWPDsnquWNnEZec3UhcLEQkFiIQCBM3Gf0F09Y+x/9gIB46NETCojoaoiQTJOUhmsiQzORZVR1jRXMPK5hrisRDpbI5M1lFXFSao/S6lApjZNufcpil/NlNQm9ly4BvAEsABdzrn7jjda7wU1Gcjlcnxqf/exY6uQboHxzg6mOTP3raO9115/qTj3n/XVra+dJyaaJCaaIiaSIjqaJDqSJDm2ihrlsRZvbiWxpoIx4ZTHBtJ0tk3xp4jQ+w+PETfSIo1S+Osb6ujoSrCj17o4bmugSnbtDgeZV1bHc8e7Ke/MDNySV2UezZfxcrmmvHjeoeTPLr7KI8938Pje3porY/xm1ev5OaN7VRN2DR4KJFmV/cQz3UNcKh/jGQmSyKdo2coye7DgxwZPBHkdbEQbQ1VtMSjNNVECASMp1/uY/+x0fn83z7J6sW1fOV9r2LV4tqZDxYpY2cb1K1Aq3PuaTOLA9uAm51zO6d7jV+CupS6B8bo2N9HNpe/PsGAcUl7Pec3VWNmZLI5ntx3nJ++2Mu7XrV8UkifzDk3Y3fPdI6PpOgdTtJaHxtfjfBkRwYTPNc5wFg6SzqbI5XJkXOQdQ6co7W+ihXNNSxfVEXAjNFklpFUhoAZ0VCAcCjAseEk+3pH2N87wmg6SyQYIOcc//ijl0iks3z23Rt46yWtZ3QOIuXgrIJ6ijfbAnzJOffQdMcoqGW+dA+M8bvfeppfHOznN16zgo9du4b6qql/YYiUs9MF9ZxGfZjZCuBSYOsUP9tsZh1m1tHT03Mm7RQ5RWt9Fd/5nSv5wFXn8/Un9vOmzz7GN5/YTyabK3XTRM6ZWVfUZlYL/Aj4pHPuu6c7VhW1LITtXQN84r928vOXjnP1qib+5bevLHWTRObNWVfUZhYG/gP4l5lCWmShXNxez93/60o+fM1qfrr3GC8fGyl1k0TOiRmD2vJ3oe4CdjnnPrfwTRKZnpnx7lctA+ChnVoQSyrDbCrqq4H3A282s2cLXzcucLtEprV8UTUXLo3zgx0KaqkMM06Bc879BNCMA/GU69Yt4UuP7uX4SIpFNZFSN0dkQfl+rQ/xp+vWLyXn4GGtBy4VQEEtZWl9Wx1t9TH1U0tFUFBLWTIz3rJuCT/e00sinZ35BSJlTEEtZevadUsYS2f5yZ7eUjdFZEEpqKVsXbGyiXg0pO4P8T0FtZStSCjAGy9czAPPdfPIboW1+JeCWsra71+7hvbGKn7rax38/neeZWA0PfOLRMqMglrK2srmGu770Gv58JtXseXZQ1z7+R/xoEf2sBSZLwpqKXuRUIDfv24tW267mubaKL/7rW383re2cXQwUeqmicwLBbX4xsXt9Wz50NX80Q1reXj3UW78wo8ZSWZK3SyRs6agFl8JBwP87zeu4jO/+kp6h1O8vIDbhImcKwpq8aXi7vM9w8kZjhTxPgW1+FJLbQxA/dTiCwpq8aWWeBRQRS3+oKAWX6qKBIlHQxwdVFBL+VNQi2+1xKOqqMUXFNTiWy3xKD1DCmopfwpq8S0FtfiFglp8S0EtfqGgFt9aHI8xnMwwmtLsRClvCmrxrfEheqqqpcwpqMW3FNTiFwpq8a3FCmrxCQW1+Faxoj6qoJYyp6AW31pUHSEYMFXUUvYU1OJbgYDRXBvh6JAWZpLypqAWX9NYavEDBbX4Wkut1vuQ8qegFl9bHI9pBT0pewpq8bWWeJRjIymyOVfqpoicsRmD2sz+ycyOmtn2c9EgkfnUEo+SzTn6RlOlborIGZtNRf014IYFbofIgihOelH3h5SzGYPaOfc4cPwctEVk3mlLLvGDeeujNrPNZtZhZh09PT3z9bYiZ0XrfYgfzFtQO+fudM5tcs5tamlpma+3FTkrJ6aRa9KLlC+N+hBfq46EqI2GVFFLWVNQi+8t1uxEKXOzGZ53N/AEsNbMOs3sgwvfLJH50xyPagU9KWuhmQ5wzr3nXDREZKG0xKPsOjRY6maInDF1fYjvLVZFLWVOQS2+1xKPapNbKWsKavG9ltr8EL0jmp0oZUpBLb63YXkDZvD1n+0vdVNEzoiCWnxvzZI477vifL7xxH62dw2Uujkic6aglorwh9evZVFNlD+99zkteSplR0EtFaG+Ksz//ZWL+EXnAN9+8kCpmyMyJwpqqRjv2NjGa17RxGce3E2vVtOTMqKglophZvzlTesZSmT47tOdpW6OyKwpqKWirFkSZ+PyBu595lCpmyIyawpqqTjvvLSdXd2D7D6saeVSHhTUUnHe9spWQgHj3me6St0UkVlRUEvFaaqN8oY1LWx55hA5DdWTMqCglop086XtHB5M8POXjpW6KSIzUlBLRbp23RJqoyF1f0hZUFBLRYqFg7z14qX89/bDJNLZUjdH5LQU1FKx3nlpO8PJDN/7hYbqibcpqKViXXlBE+vb6vjiI3tJZ3Olbo7ItBTUUrECAeMPrlvDgeOj/FuHZiqKdymopaK9ae1iLjuvgS8+skd91eJZCmqpaGbGH163lu6BBN/eqlX1xJsU1FLxXrOqmde8ookvP7ZX+yqKJymoRYA/uG4tvcMp/nzLDpzTbEXxFgW1CPCq8xv58DWr+fdtnXzyv3YprMVTQqVugIhXfOwtqxkcS/PVn+yjsSbCbW9aVeomiQAKapFxZsafv20dA2Np/vb7z2MGv/eGV2BmpW6aVDgFtcgEgYDxmXe9kmzO8ZkHn2fv0WH++p2XEAsHS900qWAKapGThIMB7rh1I6sW1/K5h15gf+8IX37vq1haHyt106RC6WaiyBTMjA9fs5qvvPcydnUP8Ya/fZS/fmAXx0dSpW6aVCBV1CKn8dZLWlnfVs/f//AF/t+PX+LbWw9w08Y2Xr+6hate0UR9VficteXAsVF+sreXJ146xvauAd69aZn60CuEzWYYkpndANwBBIGvOuc+fbrjN23a5Do6OuanhSIesefIEF98ZC8P7zrCSCpLMGBc1BrnkvZ6Lm6vZ82SOO0NVSypixEMzF94bu8a4AsP7+EHO48AsDgepb2ximcO9HPr5cv5q5svJhysrH8c942kGBhLc35T9Wl/UTnn6BtNc2QwwfGRFPVVYZbWx1hUHSEwj9doPpjZNufcpil/NlNQm1kQeAG4FugEngLe45zbOd1rFNTiZ+lsjmcO9PPjPT08faCP5zoHGEycmNEYDBiL41EaqyMsqonQWBOhLhaivipMPBamOhKkKhIkFg4SDth4qA8lMgyMpRlMpBlNZRlJZni5UEXXxUL8xtUrecfGNi5orgHg8w+9wBce2cvr17Twe294xaQ2moGRvzkaMAiYkXOQyeZIZx3pbI5kJkcqmyOTzZHNObI5R86Bw1GMBSu8NmAQCgQIBY1wMEA4GCAayn9VR0LURIPURkNEQ0Gi4QCRYGDOQeicI511ZHI5cg4CBoZxbCTJ9q4BtncNsuPQALu6hzg8mACgqSbCq1cu4rLzGllSH6O5JkIwYHS83MdP9/by9IE+EulTV0aMBANsPK+B161q5jWrmsk5R2ffKIf6EzTXRrikvYHVS2rn9Aswl3MkMzmqImd24/lsg/oq4C+dc9cXHt8O4Jz71HSvUVBLJXHOcfD4GC/1DnOoP8Gh/jEODyboG0lxfDRF/2iawbE0A2NpMrPco7EqHKQmGqQuFuZ/XNbOB16zgrrYqd0s33nqILff+xzZMtj7MRgwgmYUC2AzcA5y7sQviZlev6qllota41zUWkc8Fqbj5eNsfek4Xf1jpxx/UWsdV6xcxHmLqllaH6OxOsLAWIojg0kOHh/l5/uOsb1r+p3oo6EATTURgkEjHAiQyuYYSWYYSWZxOGLhIFWF0UAjyQyj6SyL41G2/slbzuj/z+mCejZ91O3AwQmPO4ErpviQzcBmgPPOO+8MmilSnsyM85qqOa+p+rTHOedIpHOMpjKMprIk0lmyzpHJ5hMqPqHqnm3XyS2XL+fVKxfRPZA48Tk4Cv/hHGSdI1coyCKFajgcNCKFijgUCORDNGAECkFq4+914j0y2RyZXL4aT2XyX8lMPrxGU1mGk5l8lZ7JkcxkmVgDOufIOkc2N7l9xYo9aPnPDweNUDBAYDzE8/9fLm6v58Kl8VOGSf7aFfms6R9N0TucpGcoRSKdZcPyBhbVRGb8/3dsOMlT+48TCwdZ1lhNe0MVhwcT/LKznx2HBukbSZHJOVLZHJFggJpokJpoiIAZiXT+GjoHNdEQNdEQjdULc89iNhX1u4AbnHO/XXj8fuAK59yHpnuNKmoRkbk5XUU9mw6YLmD5hMfLCs+JiMg5MJugfgpYbWYrzSwC3Arct7DNEhGRohn7qJ1zGTP7EPB98sPz/sk5t2PBWyYiIsAsJ7w45x4AHljgtoiIyBQqa5S8iEgZUlCLiHicglpExOMU1CIiHjerRZnm/KZmPcDLZ/jyZqB3HptTDirxnKEyz7sSzxkq87znes7nO+dapvrBggT12TCzjulm5/hVJZ4zVOZ5V+I5Q2We93yes7o+REQ8TkEtIuJxXgzqO0vdgBKoxHOGyjzvSjxnqMzznrdz9lwftYiITObFilpERCZQUIuIeJxngtrMbjCz581sr5n9canbs1DMbLmZPWpmO81sh5l9pPD8IjN7yMz2FP5sLHVb55uZBc3sGTO7v/B4pZltLVzzfy0so+srZtZgZv9uZrvNbJeZXeX3a21mHyv83d5uZnebWcyP19rM/snMjprZ9gnPTXltLe8LhfP/pZldNpfP8kRQFzbQ/QfgrcA64D1mtq60rVowGeAPnHPrgCuB2wrn+sfAw8651cDDhcd+8xFg14THfwN83jm3CugDPliSVi2sO4AHnXMXAhvIn79vr7WZtQMfBjY55y4mvzTyrfjzWn8NuOGk56a7tm8FVhe+NgNfmdMnOedK/gVcBXx/wuPbgdtL3a5zdO5byO/w/jzQWniuFXi+1G2b5/NcVviL+2bgfvLb8vUCoan+DvjhC6gH9lG4aT/hed9ea07ssbqI/DLK9wPX+/VaAyuA7TNdW+AfgfdMddxsvjxRUTP1BrrtJWrLOWNmK4BLga3AEudcd+FHh4ElJWrWQvl74I+AXOFxE9DvnMsUHvvxmq8EeoB/LnT5fNXMavDxtXbOdQGfBQ4A3cAAsA3/X+ui6a7tWWWcV4K64phZLfAfwEedc5P2rHf5X7m+GTdpZm8DjjrntpW6LedYCLgM+Hb7d+QAAAGOSURBVIpz7lJghJO6OXx4rRuBd5D/JdUG1HBq90BFmM9r65WgrqgNdM0sTD6k/8U5993C00fMrLXw81bgaKnatwCuBm4ys/3APeS7P+4AGsysuMuQH695J9DpnNtaePzv5IPbz9f6LcA+51yPcy4NfJf89ff7tS6a7tqeVcZ5JagrZgNdMzPgLmCXc+5zE350H/Drhe9/nXzftS845253zi1zzq0gf20fcc69F3gUeFfhMF+dM4Bz7jBw0MzWFp66BtiJj681+S6PK82suvB3vXjOvr7WE0x3be8DPlAY/XElMDChi2Rmpe6Mn9C5fiPwAvAi8Kelbs8Cnudryf9z6JfAs4WvG8n32T4M7AF+CCwqdVsX6PzfCNxf+P4C4ElgL/BvQLTU7VuA890IdBSu938CjX6/1sDHgd3AduCbQNSP1xq4m3w/fJr8v54+ON21JX/z/B8K+fYc+VExs/4sTSEXEfE4r3R9iIjINBTUIiIep6AWEfE4BbWIiMcpqEVEPE5BLSLicQpqERGP+/9NI9qzd1LGKQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gIaPMvXe2wrS"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}